package edu.gatech.doi.spamframework.parsers;
import javax.swing.text.html.*;
import javax.swing.text.html.HTMLEditorKit.Parser;
import javax.swing.text.AttributeSet;
import javax.swing.text.BadLocationException;
import javax.swing.text.Element;
import javax.swing.text.ElementIterator;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Comment;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Node;

import edu.gatech.doi.spamframework.core.HtmlDoc;

import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
/** *  Extract all "img" tags from an HTML document. */
public class HtmlParser
{  
	private HashMap Md5Hm;
	private String BodyText;
	private int totalHttpRedirectionNum=0;
	private int totalNonHttpRedirectionNum=0;
	private HashMap headerHm;
	public HtmlParser()
	{
		this.Md5Hm=new HashMap();
		this.headerHm=new HashMap();
	}
	public static void main( String[] argv ) throws Exception  
	{    
//		URL url = new URL( "http://java.sun.com" );   
//		HTMLEditorKit kit = new HTMLEditorKit();   
//		Map<String, List<String>> hm= url.openConnection().getHeaderFields();
//		Iterator it1=hm.keySet().iterator();
//		while(it1.hasNext())
//		{
//			String name=(String)it1.next();
//			List value= hm.get(name);
//			System.out.println(name+":"+value);
//		}
		HtmlParser hp=new HtmlParser();
		
		String filename=System.getProperty("user.dir")+System.getProperties().getProperty("file.separator")+"datasets"+System.getProperties().getProperty("file.separator")+"WebbSpamCorpus"+System.getProperties().getProperty("file.separator")+"9edfd4948b68fbf5f0396778f07c8511_4";
		hp.parseHTMLDoc(filename);
		
//		String dir=argv[0];
//		String resultFileName=argv[1];
//		String headerFileName=argv[2];
//		
//		File file=new File(dir);
//		File[] files=file.listFiles();
//		for(int i=0;i<files.length;i++)
//		{
//			String filename=files[i].getName();
//			System.out.println(filename);
//			hp.statsticsFiles(filename);
////			hp.parseHTMLDoc(files[i].getAbsolutePath());
//		}
//		
//		hp.filesStatsToString(resultFileName);
////		hp.headerStats(headerFileName);
	}
	
	public void filesStatsToString(String filename)
	{
		File file=new File(filename);
		FileWriter fw=null;
		BufferedWriter bw=null;
		try {
			fw=new FileWriter(file);
			bw=new BufferedWriter(fw);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		//total elements in HashMap
		int size=this.Md5Hm.size();
		System.out.println("total size:"+size);
		try {
			bw.write("total size:"+size);
			bw.newLine();
			bw.write("total http redirection number:"+this.totalHttpRedirectionNum);
			bw.newLine();
			bw.write("total non http redirection number:"+this.totalNonHttpRedirectionNum);
			bw.newLine();
			bw.flush();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		Iterator it=this.Md5Hm.keySet().iterator();
		while(it.hasNext())
		{
			String tempMd5=(String) it.next();
			HtmlDoc hd=(HtmlDoc) this.Md5Hm.get(tempMd5);
			System.out.println(hd.toString());
			try {
				bw.write(hd.toString());
				bw.newLine();
				bw.flush();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		try {
			bw.close();
			fw.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	public void statsticsFiles(String filename)
	{
		String Md5Code;
		int firstOIndex=filename.indexOf("_");
		
			Md5Code=filename;
			if(firstOIndex>0)
			Md5Code=filename.substring(0, firstOIndex);
		
		
		System.out.println(Md5Code);
		
		HtmlDoc hd;
		
		if(this.Md5Hm.containsKey(Md5Code))
		{
			hd=(HtmlDoc) this.Md5Hm.get(Md5Code);
			hd.setTotalNum(hd.getTotalNum()+1);
		}
		else
		{
			hd=new HtmlDoc();
			hd.setMd5code(Md5Code);
			hd.setTotalNum(1);
			hd.setHttpRedirectionCount(0);
			hd.setNonHttpRedirectionCount(0);
		}
//		hd.setTotalNum();
		//redirection
		if(filename.contains("redirection"))
		{
			if(filename.contains("HTTP"))
			{
				hd.setHttpRedirectionCount(hd.getHttpRedirectionCount()+1);
				this.totalHttpRedirectionNum++;
			}else
			{
				hd.setNonHttpRedirectionCount(hd.getNonHttpRedirectionCount()+1);
				this.totalNonHttpRedirectionNum++;
			}
		}
	
			this.Md5Hm.put(Md5Code, hd);
	
		
	}
	
	
	
	
	public static String html2text(String html) {    
		return Jsoup.parse(html).text();
	}
	public boolean parseHTMLDoc(String filename)
	{
//		HTMLEditorKit kit = new HTMLEditorKit(); 
//		HTMLDocument doc1 = (HTMLDocument) kit.createDefaultDocument();     
//		
//		doc1.putProperty("IgnoreCharsetDirective", Boolean.TRUE);    
////		
//		InputStream in;
		try {
//			in = new FileInputStream(filename);
//			Reader HTMLReader = new InputStreamReader(in);    
//			kit.read(HTMLReader, doc1, 0);   
//			String str=doc1.getText(0, doc1.getLength());
////			
//			
			File input = new File(filename);
			Document doc = Jsoup.parse(input, "UTF-8");
			//String str1=html2text(str);
//			Document doc1=Jsoup.parse(str);
			for(org.jsoup.nodes.Element e : doc.getAllElements()){        
				for(Node n: e.childNodes()){           
					if(n instanceof Comment){                
						//System.out.println(n);  
						Attributes attrs=n.attributes();
						Iterator it=attrs.iterator();
						while(it.hasNext())
						{
							Attribute attr=(Attribute)it.next();
							String key=attr.getKey();
							String value=attr.getValue();
							//System.out.println("key:"+key+" value:"+value); 
							//if the value contains ':',then get the sub string before the ':'.
							if(value.contains(":"))
							{
								int index=value.indexOf(":");
								String strkey=value.substring(0, index).trim().toUpperCase();
								String strvalue=strkey+"_"+value.substring(index+1).trim();
								this.headerHm.put(strkey, strvalue);
								//System.out.println("strkey:"+strkey+" strvalue:"+strvalue);  
							}
							else
							{
								if(value.contains("HTTP"))
								{
									String strkey="STATUS";
									String strvalue=value.trim();
									this.headerHm.put(strkey, strvalue);
									//System.out.println("strkey:"+strkey+" strvalue:"+strvalue);
								}
							}
						}
						}         
					}     
				} 
			String str1=doc.text();
			this.BodyText=str1;
			//System.out.println("str1:"+str1);
			
			//  Get an iterator for all HTML tags. 
			doc=null;
			
			
			return true;
			//System.out.println(doc.AdditionalComments);
  
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
			
		} 
		return false;
	}
	
	public String getBodyText()
	{
		return this.BodyText;
	}
	public HashMap getHeaderMap()
	{
		return this.headerHm;
	}
	
	public void headerStats(String filename)
	{
		File file=new File(filename);
		FileWriter fw=null;
		BufferedWriter bw=null;
		try {
			fw=new FileWriter(file);
			bw=new BufferedWriter(fw);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		//total elements in HashMap
		int size=this.headerHm.size();
		System.out.println("total size:"+size);
		try {
			bw.write("total size:"+size);
			bw.newLine();
		
			bw.flush();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		Iterator it=this.headerHm.keySet().iterator();
		while(it.hasNext())
		{
			String tempname=(String) it.next();
			Integer value=(Integer) this.headerHm.get(tempname);
			System.out.println(value.intValue());
			try {
				bw.write(tempname+" "+value.intValue());
				bw.newLine();
				bw.flush();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		try {
			bw.close();
			fw.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	
	
	
}
